import hashlib
import urllib
from urlparse import urlparse
from string import lower
#import sys

def normalize(url):
    t = urlparse(lower(url))
    u = ""
    if t.scheme == '':
        t = urlparse("http://" + t.path)
    u += t.scheme + "://" + t.netloc
    
    if t.path != '':
        path = t.path
        path = path.strip('/')
        u += "/" + path
    return u.rstrip('/')

def generate_hash(url):
    u = urlparse(normalize(url))
    url = "%s%s" % (u.hostname, u.path)
    return hashlib.md5(url).hexdigest()

#print normalize(sys.argv[1])
#print get_hash(sys.argv[1])
